"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 极致RTO-硬件故障类
Case Name   : 开启极致RTO，业务压力后进行数据导入导出过程中，主机磁盘满故障后恢复；重新导出完成后，关闭极致RTO，主备数据一致
Create At   : 2020/10/15
Owner       : @peilinqian
Description :
    1、开启极致RTO
    2、进行gs_dump
    3、进行gs_restore
    4、注入磁盘满故障
    5、等待10S，查询主备状态
    6、restore进程是否异常
    7、等待10S，恢复主机故障
    8、重启数据库
    9、查询主备状态
    10、再次进行restore
    11、restore执行完成后，关闭极致RTO
    12、查询主备数据是否一致
Expect      :
    1、开启极致RTO成功
    2、进行gs_dump功能正常
    3、进行gs_restore运行开始
    4、注入磁盘满故障成功
    5、等待10S，查询主备状态，主机stop
    6、restore进程查询为空、执行异常
    7、等待10S，清理磁盘成功
    8、重启数据库成功
    9、查询主备状态正常
    10、再次进行restore正常
    11、restore执行完成后，关闭极致RTO正常
    12、查询主备数据一致
    13、执行TPCC正常
History     :
    modified：2021-9-3 by 5328113;注入磁盘满故障，查询状态断言删除，只保证restore结果失败即可
    modified：2021-7-19 by 5328113;注入磁盘满故障后，restore结果增加断言条件；
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified：2021-3-9 by 5328113;修改dump文件路径；
    modified：2021-2-5 by 5328113;guc参数还原时指定还原值
    modified：2020-12-14;修改restore指定新数据库，以防用例失败影响tpcc库及tpcc运行；
    modified：2020-12-14;restore注入故障后，再次restore偶现连不上主机问题，tpcc执行完成后增加等待同步
    modified：2021-1-28 by 5328113;删除start成功断言，删除不必要tpcc
"""

import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import *
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(
            '----Opengauss_Reliability_RTO_Hardware_Disk_Case004:初始化----')

        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.tpcc_path = macro.TPCC_PATH
        dump_file_name = 'test_pid.tar'
        self.parent_path = os.path.dirname(macro.DB_INSTANCE_PATH)
        self.file_name = os.path.join(self.parent_path, dump_file_name)

        self.log.info("-----创建restore数据库----")
        self.new_db_name = 'restore_db'
        self.create_db_cmd = f'drop database if exists {self.new_db_name};' \
            f'create database {self.new_db_name};'
        self.drop_db_cmd = f'drop database {self.new_db_name};'
        create_db_result = self.pri_sh.executDbSql(self.create_db_cmd)
        self.log.info(create_db_result)
        self.assertIn(self.constant.CREATE_DATABASE_SUCCESS_MSG,
                      create_db_result)
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            self.log.info('----1、开启极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----2、进行gs_dump----')
            dump_result = self.pri_sh.dumpFile(self.file_name,
                                               get_detail=False)
            self.log.info(dump_result)
            self.assertTrue(dump_result)

            self.log.info('----3、进行gs_restore----')
            restore_thread = ComThread(self.pri_sh.restoreFile,
                                       args=(self.file_name, '', True,
                                             self.new_db_name,))
            restore_thread.setDaemon(True)
            restore_thread.start()
            time.sleep(3)

            self.log.info('----4、注入磁盘满故障----')
            self.log.info("-----------获取主机磁盘信息 -----------")
            df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
            self.log.info(df_cmd)
            df_result = self.pri_root.sh(df_cmd).result()
            self.log.info(df_result)
            self.DISK_NAME = df_result.splitlines()[-1].split()[0].strip()

            self.log.info("-----------注入磁盘满故障 -----------")
            inject_cmd = f'rfile_full (diskname) values ({self.DISK_NAME})'
            inject_result = self.com.cfe_inject(self.pri_root, inject_cmd)
            self.log.info(inject_result)
            success_msg = self.constant.CFE_DISK_FULL_SUCCESS_MSG
            self.assertIn(success_msg, inject_result)

            self.log.info("-----------获取主机磁盘占用情况 -----------")
            df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
            self.log.info(df_cmd)
            df_result = self.pri_root.sh(df_cmd).result()
            self.log.info(df_result)
            self.USED_PERCENT = df_result.splitlines()[-1].split()[4].strip()
            self.assertTrue(self.USED_PERCENT == '100%')

            self.log.info('----5、等待10S，查询主备状态----')
            time.sleep(10)
            self.sta_1_sh.getDbClusterStatus('status')

            self.log.info('----6、restore进程是否异常----')
            restore_status = self.pri_dbuser.sh(
                'ps ux | grep gs_restore | grep -v grep').result()
            self.log.info(restore_status)
            # 等待restore执行完成
            self.log.info('----restore执行结果----')
            restore_thread.join()
            restore_result = restore_thread.get_result()
            self.log.info(restore_result)
            self.assertTrue(self.constant.RESTORE_FAIL_MSG in restore_result
                            or self.constant.DISK_FULL_MSG in restore_result)

            self.log.info('----7、等待10S，恢复主机故障----')
            time.sleep(10)
            self.log.info("-----------清除磁盘满故障 -----------")
            clean_cmd = f'rfile_full where(diskname={self.DISK_NAME})'
            clean_result = self.com.cfe_clean(self.pri_root, clean_cmd)
            self.log.info(clean_result)
            self.assertIn(self.constant.CFE_DISK_CLEAN_SUCCESS_MSG,
                          clean_result)
            self.log.info("-----------获取主机磁盘占用情况 -----------")
            df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
            self.log.info(df_cmd)
            df_result = self.pri_root.sh(df_cmd).result()
            self.log.info(df_result)
            self.USED_PERCENT = df_result.splitlines()[-1].split()[4].strip()
            self.assertTrue(self.USED_PERCENT != '100%')

            self.log.info('----8、重启数据库----')
            self.pri_sh.startDbCluster()

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.max_time)
            self.log.info('----9、查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----10、再次进行restore----')
            self.log.info('----刪除之前的restore库----')
            drop_db_result = self.pri_sh.executDbSql(self.drop_db_cmd)
            self.log.info(drop_db_result)
            self.assertIn(self.constant.drop_database_success_msg,
                          drop_db_result)
            self.log.info('----刪除之前的restore库----')
            create_db_result = self.pri_sh.executDbSql(self.create_db_cmd)
            self.log.info(create_db_result)
            self.assertIn(self.constant.CREATE_DATABASE_SUCCESS_MSG,
                          create_db_result)

            self.log.info('----再次进行restore----')
            restore_result = self.pri_sh.restoreFile(self.file_name,
                                                     '',
                                                     True,
                                                     self.new_db_name)
            self.assertNotIn(self.constant.RESTORE_FAIL_MSG, restore_result)

            self.log.info('----同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.max_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

        finally:
            self.log.info('----11、restore执行完成后，关闭极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            # 校验备机是否完成数据同步
            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2)

            self.log.info('----12、查询主备数据是否一致----')
            nodes_tuple = (self.pri_dbuser,
                           self.sta1_dbuser,
                           self.sta2_dbuser)
            flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
            self.assertTrue(flag)

            table_dict = self.com.format_sql_result(
                self.pri_sh.executDbSql(r'\d'))
            table_name = table_dict.get('Name')
            for name in table_name:
                select_sql = f'select count(*) from {name};'
                flag = self.com.check_data_sample_by_all(select_sql,
                                                         *nodes_tuple)
                self.assertTrue(flag)

    def tearDown(self):
        self.log.info('----删除tar文件----')
        self.pri_dbuser.sh(f'rm -rf {self.file_name}').result()
        self.log.info(self.pri_dbuser.sh(f'ls -l {self.file_name}').result())

        self.log.info("----删除restore库----")
        drop_db_result = self.pri_sh.executDbSql(self.drop_db_cmd)
        self.log.info(drop_db_result)

        df_cmd = f"df -h {macro.DB_INSTANCE_PATH}"
        self.log.info(df_cmd)
        df_result = self.pri_root.sh(df_cmd).result()
        self.log.info(df_result)
        self.USED_PERCENT = df_result.splitlines()[-1].split()[4].strip()
        if self.USED_PERCENT == '100%':
            self.log.info('----异常中断，磁盘故障未恢复处理----')
            clean_disk_cmd = f'rfile_full  where(diskname={self.DISK_NAME})'
            clean_disk_result = self.com.cfe_clean(self.pri_root,
                                                   clean_disk_cmd)
            self.log.info(clean_disk_result)

        self.log.info('----校验刷脏页是否完成----')
        self.pri_sh.check_whether_remain_dirty_page()

        self.log.info(
            '----Opengauss_Reliability_RTO_Hardware_Disk_Case004:执行完毕----')
